{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from malaya.train.model.bigbird import modeling, utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_config = {\n",
    "    'attention_probs_dropout_prob': 0.1,\n",
    "    'hidden_act': 'gelu',\n",
    "    'hidden_dropout_prob': 0.1,\n",
    "    'hidden_size': 512,\n",
    "    'initializer_range': 0.02,\n",
    "    'intermediate_size': 2048,\n",
    "    'max_position_embeddings': 2048,\n",
    "    'max_encoder_length': 1024,\n",
    "    'max_decoder_length': 1024,\n",
    "    'num_attention_heads': 8,\n",
    "    'num_hidden_layers': 6,\n",
    "    'type_vocab_size': 2,\n",
    "    'scope': 'bert',\n",
    "    'use_bias': True,\n",
    "    'rescale_embedding': False,\n",
    "    'vocab_model_file': None,\n",
    "    'attention_type': 'block_sparse',\n",
    "    'block_size': 16,\n",
    "    'num_rand_blocks': 3,\n",
    "    'vocab_size': 32000,\n",
    "    'couple_encoder_decoder': False,\n",
    "    'beam_size': 1,\n",
    "    'alpha': 0.0,\n",
    "    'label_smoothing': 0.1,\n",
    "    'norm_type': 'postnorm',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sentencepiece as spm\n",
    "\n",
    "vocab = 'sp10m.cased.translation.model'\n",
    "sp = spm.SentencePieceProcessor()\n",
    "sp.Load(vocab)\n",
    "\n",
    "class Encoder:\n",
    "    def __init__(self, sp):\n",
    "        self.sp = sp\n",
    "    \n",
    "    def encode(self, s):\n",
    "        return self.sp.EncodeAsIds(s) + [1]\n",
    "    \n",
    "    def decode(self, ids, strip_extraneous=False):\n",
    "        return self.sp.DecodeIds(list(ids))\n",
    "    \n",
    "encoder = Encoder(sp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = modeling.TransformerModel(bert_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = tf.placeholder(tf.int32, [None, None])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/malaya/Malaya/malaya/train/model/bigbird/modeling.py:226: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((<tf.Tensor 'bert/log_probs:0' shape=(?, 1024) dtype=float32>,\n",
       "  <tf.Tensor 'bert/logits:0' shape=(?, 1024, 32000) dtype=float32>,\n",
       "  <tf.Tensor 'bert/while/Exit_1:0' shape=(?, 1024) dtype=int32>),\n",
       " <tf.Tensor 'bert/encoder/layer_5/output/LayerNorm/batchnorm/add_1:0' shape=(?, 1024, 512) dtype=float32>)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r = model(X, training = False)\n",
    "r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'logits:0' shape=(?, 1024) dtype=int32>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits = tf.identity(r[0][2], name = 'logits')\n",
    "logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'bigbird-base-ms-en/model.ckpt-435000'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ckpt_path = tf.train.latest_checkpoint('bigbird-base-ms-en')\n",
    "# ckpt_path = 'checkpoint/model.ckpt-55000'\n",
    "ckpt_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from bigbird-base-ms-en/model.ckpt-435000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from bigbird-base-ms-en/model.ckpt-435000\n"
     ]
    }
   ],
   "source": [
    "saver = tf.train.Saver()\n",
    "saver.restore(sess, ckpt_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "pad_sequences = tf.keras.preprocessing.sequence.pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "\n",
    "def cleaning(string):\n",
    "    return re.sub(r'[ ]+', ' ', unidecode(string.replace('\\n', ' '))).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'KUALA LUMPUR: Perseteruan antara dua bekas Perdana Menteri, Tun Dr Mahathir Mohamad dan Datuk Seri Najib Tun Razak belum ada penghujungnya dengan masing-masing berbalas kenyataan di media sosial. Selepas Najib menyanggah kenyataan tidak campur tangan dalam badan kehakiman negara dan mentertawakannya, Dr Mahathir membalasnya dengan meminta Ahli Parlimen Pekan itu memberi perhatian kepada kes 1Malaysia Development Berhad (1MDB). Dr Mahathir juga secara sinis berkata, jika Najib boleh mengingati isu yang berlaku pada 1987 dan 1988 - isu pemilihan UMNO dan pengharaman parti itu, Najib juga boleh mengingati peristiwa dia ingin mandikan keris dengan darah. \"Saya rasa Najib tak payah campur tangan dengan tuduhan terhadap saya. Dia harus fokus kes curi duit rakyat berbilion-bilion dalam 1MDB. \"Dia juga perlu bagi perhatian saman Tommy Thomas yang kait dia dengan pembunuhan Altantuya. Lagipun, kalau isu yang berlaku 1987/88, Najib boleh ingat (peristiwa) yang dia \\'hunus\\' keris,\" kata Dr Mahathir. Sebelum ini, Najib menyanggah dakwaan Dr Mahathir yang mendakwa pembatalan pendaftaran UMNO pada 1998 sebagai bukti tidak campur tangan dalam badan kehakiman negara. Menyokong hujahnya, Najib berkongsi apa yang berlaku pada tahun tersebut hingga menyebabkan UMNO diharamkan dan tertubuhnya UMNO baharu.'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = \"\"\"\n",
    "KUALA LUMPUR: Perseteruan antara dua bekas Perdana Menteri, Tun Dr Mahathir Mohamad dan Datuk Seri Najib Tun Razak belum ada penghujungnya dengan masing-masing berbalas kenyataan di media sosial.\n",
    "Selepas Najib menyanggah kenyataan tidak campur tangan dalam badan kehakiman negara dan mentertawakannya, Dr Mahathir membalasnya dengan meminta Ahli Parlimen Pekan itu memberi perhatian kepada kes 1Malaysia Development Berhad (1MDB).\n",
    "Dr Mahathir juga secara sinis berkata, jika Najib boleh mengingati isu yang berlaku pada 1987 dan 1988 - isu pemilihan UMNO dan pengharaman parti itu, Najib juga boleh mengingati peristiwa dia ingin mandikan keris dengan darah.\n",
    "\"Saya rasa Najib tak payah campur tangan dengan tuduhan terhadap saya. Dia harus fokus kes curi duit rakyat berbilion-bilion dalam 1MDB.\n",
    "\"Dia juga perlu bagi perhatian saman Tommy Thomas yang kait dia dengan pembunuhan Altantuya. Lagipun, kalau isu yang berlaku 1987/88, Najib boleh ingat (peristiwa) yang dia 'hunus' keris,\" kata Dr Mahathir.\n",
    "Sebelum ini, Najib menyanggah dakwaan Dr Mahathir yang mendakwa pembatalan pendaftaran UMNO pada 1998 sebagai bukti tidak campur tangan dalam badan kehakiman negara.\n",
    "Menyokong hujahnya, Najib berkongsi apa yang berlaku pada tahun tersebut hingga menyebabkan UMNO diharamkan dan tertubuhnya UMNO baharu.\n",
    "\"\"\"\n",
    "cleaning(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "string3 = \"\"\"\n",
    "Penubuhan universiti sukan seperti diutarakan Ketua Unit Sukan Kementerian Pengajian Tinggi, Dr Pekan Ramli dan disokong Pakar Pembangunan Sukan dan Reakreasi Luar, Universiti Pendidikan Sultan Idris (UPSI), Prof Dr Md Amin Md Taaf seperti disiarkan akhbar ini, memberikan sinar harapan kepada kewujudan institusi sedemikian.\n",
    "\n",
    "Ia menjadi impian atlet negara untuk mengejar kejayaan dalam bidang sukan dan kecemerlangan dalam akademik untuk menjamin masa depan lebih baik apabila bersara daripada arena sukan kelak.\n",
    "\n",
    "Pelbagai pandangan, idea, kaedah, bukti dan cadangan dilontarkan pakar berikutan pentingnya universiti sukan yang akan memberi impak besar sama ada pada peringkat kebangsaan mahupun antarabangsa.\n",
    "\n",
    "Negara lain sudah lama meraih laba dengan kewujudan universiti sukan seperti China, Korea, Japan, Taiwan, India dan Vietnam. Mereka menghasilkan atlet universiti yang mempamerkan keputusan cemerlang pada peringkat tinggi seperti Sukan Olimpik, Kejohanan Dunia dan Sukan Asia.\n",
    "\n",
    "Justeru, kejayaan mereka perlu dijadikan rujukan demi memajukan sukan tanah air. Jika kita merujuk pendekatan Asia, kewujudan universiti sukan penting dan memberi kesan positif dalam melonjakkan prestasi sukan lebih optimum.\n",
    "\n",
    "Namun, jika kita melihat pendekatan Eropah, universiti sukan bukan antara organisasi atau institusi penting yang diberi perhatian dalam menyumbang kepada pemenang pingat.\n",
    "\n",
    "Antara isu dalam universiti sukan ialah kos tinggi, lokasi, prasarana sukan, pertindihan kursus dengan universiti sedia ada dan impak terhadap dunia sukan negara hingga mengundang persoalan kewajaran dan kerelevanan penubuhannya.\n",
    "\n",
    "Namun sebagai bekas atlet memanah negara dan Olympian (OLY) di Sukan Olimpik 2004 di Athens, Greece serta bekas pelajar Sekolah Sukan Bukit Jalil hingga berjaya dalam dunia akademik, saya mendapati terdapat beberapa faktor sering menjadi halangan dalam rutin harian mereka.\n",
    "\n",
    "Antaranya, faktor masa yang terpaksa bergegas menghadiri kuliah selepas tamat sesi latihan yang mengambil masa 15 hingga 20 minit dengan menunggang motosikal; kereta (20-30 minit) atau pengangkutan disediakan Majlis Sukan Negara (MSN) ke Universiti Putra Malaysia (UPM).\n",
    "\n",
    "Jika mereka menuntut di Universiti Teknologi MARA (UiTM) atau Universiti Malaya (UM), ia mungkin lebih lama.\n",
    "\n",
    "Walaupun di universiti tersedia dengan kemudahan kolej dan kemudahan sukan, mereka memilih pulang ke MSN untuk menjalani latihan bersama pasukan dan jurulatih di padang atau gelanggang latihan rasmi.\n",
    "\n",
    "Ini berlanjutan selagi bergelar atlet negara yang perlu memastikan prestasi sentiasa meningkat dari semasa ke semasa tanpa mengabaikan tugas sebagai pelajar.\n",
    "\n",
    "Alangkah baiknya jika sebahagian Sekolah Sukan Bukit Jalil itu sendiri dijadikan Kolej atau Universiti Sukan Malaysia kerana lengkap dari segi kemudahan prasarana sukannya dan proses pengajaran dan pembelajaran (PdP) dalam bidang Sains Sukan, Kejurulatihan, Pendidikan Jasmani dan setaraf dengannya.\n",
    "\n",
    "Pengambilan setiap semester pula hanya terhad kepada atlet berstatus kebangsaan dan antarabangsa sahaja supaya hasrat melahirkan lebih ramai atlet bertaraf Olimpik mudah direalisasikan.\n",
    "\n",
    "Contohnya, bekas atlet lompat bergalah negara, Roslinda Samsu yang juga pemenang pingat perak Sukan Asia Doha 2006 dan Penerima Anugerah Khas Majlis Anugerah Sukan KPT 2012, terpaksa mengambil masa lebih kurang sembilan tahun untuk menamatkan ijazah Sarjana Muda Pendidikan Jasmani di UPM sepanjang 14 tahun terbabit dalam sukan olahraga.\n",
    "\n",
    "Sepanjang tempoh bergelar atlet kebangsaan dan mahasiswa, beliau juga memenangi pingat Emas Sukan SEA empat siri berturut-turut pada 2005, 2007, 2009 dan 2011.\n",
    "\n",
    "Begitu juga atlet kebangsaan seperti Leong Mun Yee (UPM); Pandalela Renong (UM); Bryan Nickson Lomas (UM); Cheng Chu Sian (UPM); Marbawi Sulaiman (UiTM) dan Norasheela Khalid (UPM).\n",
    "\n",
    "Jika disenaraikan, mungkin lebih ramai lagi. Namun, pernah terlintas di fikiran mengapa hanya atlet dari sukan terjun yang dapat memenangi pingat di Sukan Olimpik? Bagaimana dengan atlet lain yang juga layak secara merit? Apakah kekangan atau masalah dihadapi sebagai atlet dan mahasiswa?\n",
    "\n",
    "Adakah kewujudan universiti sukan akan memberi impak besar kepada kemajuan sukan negara? Jika dirancang dan diatur dengan cekap dan sistematik, ia perkara tidak mustahil dicapai.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded = encoder.encode(f'{cleaning(string3)}') + [1]\n",
    "s = pad_sequences([encoded], padding='post', maxlen = 1024)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 14 s, sys: 775 ms, total: 14.8 s\n",
      "Wall time: 12.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "l = sess.run(r[0][2], feed_dict = {X: s})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Establishment of sports universities as stated by the Head of the Sports Unit of the Ministry of Higher Education, Dr Pekan Ramli and supported by the Specialist for Sports Development and Outdoor Recreation, Sultan Idris University of Education (UPSI), Prof Dr Md Amin Md Taaf as published in this newspaper, gives a ray of hope to the existence of such institutions. It is the dream of national athletes to pursue success in sports and academic excellence to ensure a better future when they retire from the sports arena in the future. Various views, ideas, methods, evidence and suggestions were put forward by experts due to the importance of sports universities that will have a huge impact either at the national or international levels. Other countries have long made a profit with the existence of sports universities such as China, Korea, Japan, Taiwan, India and Vietnam. They produce university athletes who show excellent results at high levels such as the Olympic Games, World Championships and Asian Games. Therefore, their success should be used as a reference in order to develop the country's sports. If we refer to the Asian approach, the existence of sports universities is important and has a positive impact in boosting sports performance is more optimal. However, if we look at the European approach, sports universities are not between important organizations or institutions that are given attention in contributing to the medal winners. Among the issues in sports universities are high costs, location, sports infrastructure, duplication of courses with existing universities and the impact on the world of national sports to the question of the legitimacy and relevance of its establishment. However, as a former national archery athlete and Olympian (OLY) at the 2004 Olympic Games in Athens, Greece and former students of Bukit Jalil Sports School to succeed in the academic world, I found that there are several factors that are often obstacles in their daily routine. Among them, the time factor that had to rush to attend lectures after the training session which took 15 to 20 minutes by riding a motorcycle; cars (20-30 minutes) or transportation provided by the National Sports Council (MSN) to Universiti Putra Malaysia (UPM). If they study at Universiti Teknologi MARA (UiTM) or Universiti Malaya (UM), it may be longer. Although at the university there are college facilities and sports facilities, they choose to return to MSN to undergo training with teams and coaches on the field or the official training court. This continues as long as they become national athletes who need to ensure that performance is constantly increasing from time to time without neglecting their duties as students. It would be good if part of Bukit Jalil Sports School itself is used as a College or Universiti Sukan Malaysia because it is complete in terms of sports infrastructure facilities and the teaching and learning process (PdP) in the field of Sports Science, Coaching, Physical Education and its equivalent. The intake of each semester is limited to athletes with national and international status only so that the desire to produce more Olympic athletes is easily realized. For example, former national team jump athlete, Roslinda Samsu who is also the silver medalist of the Doha Asian Games 2006 and the Special Award recipient of the MOHE Sports Awards Council 2012, had to take about nine years to complete the Bachelor of Physical Education at UPM for 14 years involved in sports sports. During his tenure as a national athlete and student, he also won the Gold Games medal for four consecutive series in 2005, 2007, 2009 and 2011. Similarly, national athletes such as Leong Mun Yee (UPM); Pandalela Renong (UM); Bryan Nickson Lomas (UM); Cheng Chu Sian (UPM); Marbawi Sulaiman (UiTM) and Norasheela Khalid (UPM). If listed, there may be more people. However, it has come to mind why only athletes from the sport of diving can win medals at the Olympic Games? What about other athletes who are also eligible on merit? What are the constraints or problems faced as athletes and students? Will the existence of sports universities have a huge impact on the progress of national sports? If planned and arranged efficiently and systematically, it is impossible to achieve.\""
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder.decode([i for i in l[0].tolist() if i > 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/test-ms-en.tar.gz\n",
    "# !tar -zxf test-ms-en.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100000, 100000)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size = 24\n",
    "\n",
    "path = 'test'\n",
    "\n",
    "with open(os.path.join(path, 'left.txt')) as fopen:\n",
    "    left = fopen.read().split('\\n')\n",
    "    \n",
    "with open(os.path.join(path, 'right.txt')) as fopen:\n",
    "    right = fopen.read().split('\\n')\n",
    "    \n",
    "len(left), len(right)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded = encoder.encode(left[0]) + [1]\n",
    "s = pad_sequences([encoded], padding='post', maxlen = 1024)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[10730,\n",
       "  1142,\n",
       "  14729,\n",
       "  221,\n",
       "  13,\n",
       "  501,\n",
       "  13,\n",
       "  960,\n",
       "  4782,\n",
       "  2005,\n",
       "  10730,\n",
       "  1142,\n",
       "  14729,\n",
       "  221,\n",
       "  13,\n",
       "  501,\n",
       "  13,\n",
       "  960,\n",
       "  4782,\n",
       "  2005,\n",
       "  30,\n",
       "  29,\n",
       "  1593,\n",
       "  25,\n",
       "  21,\n",
       "  20974,\n",
       "  1724,\n",
       "  22,\n",
       "  21,\n",
       "  20209,\n",
       "  13,\n",
       "  9752,\n",
       "  1130,\n",
       "  22,\n",
       "  12816,\n",
       "  156,\n",
       "  1432,\n",
       "  9]]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = sess.run(logits, feed_dict = {X: s}).tolist()\n",
    "results = []\n",
    "for row in p:\n",
    "    results.append([i for i in row if i not in [0, 1]])\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.97402745"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensor2tensor.utils import bleu_hook\n",
    "bleu_hook.compute_bleu(reference_corpus = [encoder.encode(right[0])], \n",
    "                       translation_corpus = results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4167/4167 [2:45:03<00:00,  2.38s/it]  \n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "results = []\n",
    "for i in tqdm(range(0, len(left), batch_size)):\n",
    "    index = min(i + batch_size, len(left))\n",
    "    x = left[i: index]\n",
    "    encoded = [encoder.encode(l) + [1] for l in x]\n",
    "    batch_x = pad_sequences(encoded, padding='post', maxlen = 1024)\n",
    "    \n",
    "    p = sess.run(logits, feed_dict = {X: batch_x}).tolist()\n",
    "    result = []\n",
    "    for row in p:\n",
    "        result.append([i for i in row if i not in [0, 1]])\n",
    "    results.extend(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.37341127"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rights = [encoder.encode(r) for r in right[:len(results)]]\n",
    "bleu_hook.compute_bleu(reference_corpus = rights,\n",
    "                       translation_corpus = results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'output/model.ckpt'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'output/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['bert/embeddings/word_embeddings',\n",
       " 'bert/embeddings/position_embeddings',\n",
       " 'Placeholder',\n",
       " 'bert/encoder/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_0/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/query/bias',\n",
       " 'bert/encoder/layer_0/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/key/bias',\n",
       " 'bert/encoder/layer_0/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/value/bias',\n",
       " 'bert/encoder/layer_0/attention/self/Softmax',\n",
       " 'bert/encoder/layer_0/attention/self/Softmax_1',\n",
       " 'bert/encoder/layer_0/attention/self/Softmax_2',\n",
       " 'bert/encoder/layer_0/attention/self/Softmax_3',\n",
       " 'bert/encoder/layer_0/attention/self/Softmax_4',\n",
       " 'bert/encoder/layer_0/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_0/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_0/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_0/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_0/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_0/output/dense/kernel',\n",
       " 'bert/encoder/layer_0/output/dense/bias',\n",
       " 'bert/encoder/layer_0/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_1/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/query/bias',\n",
       " 'bert/encoder/layer_1/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/key/bias',\n",
       " 'bert/encoder/layer_1/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/value/bias',\n",
       " 'bert/encoder/layer_1/attention/self/Softmax',\n",
       " 'bert/encoder/layer_1/attention/self/Softmax_1',\n",
       " 'bert/encoder/layer_1/attention/self/Softmax_2',\n",
       " 'bert/encoder/layer_1/attention/self/Softmax_3',\n",
       " 'bert/encoder/layer_1/attention/self/Softmax_4',\n",
       " 'bert/encoder/layer_1/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_1/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_1/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_1/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_1/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_1/output/dense/kernel',\n",
       " 'bert/encoder/layer_1/output/dense/bias',\n",
       " 'bert/encoder/layer_1/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_2/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/query/bias',\n",
       " 'bert/encoder/layer_2/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/key/bias',\n",
       " 'bert/encoder/layer_2/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/value/bias',\n",
       " 'bert/encoder/layer_2/attention/self/Softmax',\n",
       " 'bert/encoder/layer_2/attention/self/Softmax_1',\n",
       " 'bert/encoder/layer_2/attention/self/Softmax_2',\n",
       " 'bert/encoder/layer_2/attention/self/Softmax_3',\n",
       " 'bert/encoder/layer_2/attention/self/Softmax_4',\n",
       " 'bert/encoder/layer_2/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_2/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_2/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_2/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_2/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_2/output/dense/kernel',\n",
       " 'bert/encoder/layer_2/output/dense/bias',\n",
       " 'bert/encoder/layer_2/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_3/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/query/bias',\n",
       " 'bert/encoder/layer_3/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/key/bias',\n",
       " 'bert/encoder/layer_3/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/value/bias',\n",
       " 'bert/encoder/layer_3/attention/self/Softmax',\n",
       " 'bert/encoder/layer_3/attention/self/Softmax_1',\n",
       " 'bert/encoder/layer_3/attention/self/Softmax_2',\n",
       " 'bert/encoder/layer_3/attention/self/Softmax_3',\n",
       " 'bert/encoder/layer_3/attention/self/Softmax_4',\n",
       " 'bert/encoder/layer_3/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_3/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_3/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_3/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_3/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_3/output/dense/kernel',\n",
       " 'bert/encoder/layer_3/output/dense/bias',\n",
       " 'bert/encoder/layer_3/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_4/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_4/attention/self/query/bias',\n",
       " 'bert/encoder/layer_4/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_4/attention/self/key/bias',\n",
       " 'bert/encoder/layer_4/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_4/attention/self/value/bias',\n",
       " 'bert/encoder/layer_4/attention/self/Softmax',\n",
       " 'bert/encoder/layer_4/attention/self/Softmax_1',\n",
       " 'bert/encoder/layer_4/attention/self/Softmax_2',\n",
       " 'bert/encoder/layer_4/attention/self/Softmax_3',\n",
       " 'bert/encoder/layer_4/attention/self/Softmax_4',\n",
       " 'bert/encoder/layer_4/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_4/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_4/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_4/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_4/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_4/output/dense/kernel',\n",
       " 'bert/encoder/layer_4/output/dense/bias',\n",
       " 'bert/encoder/layer_4/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_5/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_5/attention/self/query/bias',\n",
       " 'bert/encoder/layer_5/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_5/attention/self/key/bias',\n",
       " 'bert/encoder/layer_5/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_5/attention/self/value/bias',\n",
       " 'bert/encoder/layer_5/attention/self/Softmax',\n",
       " 'bert/encoder/layer_5/attention/self/Softmax_1',\n",
       " 'bert/encoder/layer_5/attention/self/Softmax_2',\n",
       " 'bert/encoder/layer_5/attention/self/Softmax_3',\n",
       " 'bert/encoder/layer_5/attention/self/Softmax_4',\n",
       " 'bert/encoder/layer_5/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_5/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_5/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_5/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_5/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_5/output/dense/kernel',\n",
       " 'bert/encoder/layer_5/output/dense/bias',\n",
       " 'bert/encoder/layer_5/output/LayerNorm/gamma',\n",
       " 'bert/decoder/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_0/attention/self/query/kernel',\n",
       " 'bert/decoder/layer_0/attention/self/query/bias',\n",
       " 'bert/decoder/layer_0/attention/self/key/kernel',\n",
       " 'bert/decoder/layer_0/attention/self/key/bias',\n",
       " 'bert/decoder/layer_0/attention/self/value/kernel',\n",
       " 'bert/decoder/layer_0/attention/self/value/bias',\n",
       " 'bert/while/decoder/layer_0/attention/self/Softmax',\n",
       " 'bert/decoder/layer_0/attention/output/dense/kernel',\n",
       " 'bert/decoder/layer_0/attention/output/dense/bias',\n",
       " 'bert/decoder/layer_0/attention/output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_0/attention/encdec/query/kernel',\n",
       " 'bert/decoder/layer_0/attention/encdec/query/bias',\n",
       " 'bert/decoder/layer_0/attention/encdec/key/kernel',\n",
       " 'bert/decoder/layer_0/attention/encdec/key/bias',\n",
       " 'bert/decoder/layer_0/attention/encdec/value/kernel',\n",
       " 'bert/decoder/layer_0/attention/encdec/value/bias',\n",
       " 'bert/decoder/layer_0/attention/encdec_output/dense/kernel',\n",
       " 'bert/decoder/layer_0/attention/encdec_output/dense/bias',\n",
       " 'bert/decoder/layer_0/attention/encdec_output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_0/intermediate/dense/kernel',\n",
       " 'bert/decoder/layer_0/intermediate/dense/bias',\n",
       " 'bert/decoder/layer_0/output/dense/kernel',\n",
       " 'bert/decoder/layer_0/output/dense/bias',\n",
       " 'bert/decoder/layer_0/output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_1/attention/self/query/kernel',\n",
       " 'bert/decoder/layer_1/attention/self/query/bias',\n",
       " 'bert/decoder/layer_1/attention/self/key/kernel',\n",
       " 'bert/decoder/layer_1/attention/self/key/bias',\n",
       " 'bert/decoder/layer_1/attention/self/value/kernel',\n",
       " 'bert/decoder/layer_1/attention/self/value/bias',\n",
       " 'bert/while/decoder/layer_1/attention/self/Softmax',\n",
       " 'bert/decoder/layer_1/attention/output/dense/kernel',\n",
       " 'bert/decoder/layer_1/attention/output/dense/bias',\n",
       " 'bert/decoder/layer_1/attention/output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_1/attention/encdec/query/kernel',\n",
       " 'bert/decoder/layer_1/attention/encdec/query/bias',\n",
       " 'bert/decoder/layer_1/attention/encdec/key/kernel',\n",
       " 'bert/decoder/layer_1/attention/encdec/key/bias',\n",
       " 'bert/decoder/layer_1/attention/encdec/value/kernel',\n",
       " 'bert/decoder/layer_1/attention/encdec/value/bias',\n",
       " 'bert/decoder/layer_1/attention/encdec_output/dense/kernel',\n",
       " 'bert/decoder/layer_1/attention/encdec_output/dense/bias',\n",
       " 'bert/decoder/layer_1/attention/encdec_output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_1/intermediate/dense/kernel',\n",
       " 'bert/decoder/layer_1/intermediate/dense/bias',\n",
       " 'bert/decoder/layer_1/output/dense/kernel',\n",
       " 'bert/decoder/layer_1/output/dense/bias',\n",
       " 'bert/decoder/layer_1/output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_2/attention/self/query/kernel',\n",
       " 'bert/decoder/layer_2/attention/self/query/bias',\n",
       " 'bert/decoder/layer_2/attention/self/key/kernel',\n",
       " 'bert/decoder/layer_2/attention/self/key/bias',\n",
       " 'bert/decoder/layer_2/attention/self/value/kernel',\n",
       " 'bert/decoder/layer_2/attention/self/value/bias',\n",
       " 'bert/while/decoder/layer_2/attention/self/Softmax',\n",
       " 'bert/decoder/layer_2/attention/output/dense/kernel',\n",
       " 'bert/decoder/layer_2/attention/output/dense/bias',\n",
       " 'bert/decoder/layer_2/attention/output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_2/attention/encdec/query/kernel',\n",
       " 'bert/decoder/layer_2/attention/encdec/query/bias',\n",
       " 'bert/decoder/layer_2/attention/encdec/key/kernel',\n",
       " 'bert/decoder/layer_2/attention/encdec/key/bias',\n",
       " 'bert/decoder/layer_2/attention/encdec/value/kernel',\n",
       " 'bert/decoder/layer_2/attention/encdec/value/bias',\n",
       " 'bert/decoder/layer_2/attention/encdec_output/dense/kernel',\n",
       " 'bert/decoder/layer_2/attention/encdec_output/dense/bias',\n",
       " 'bert/decoder/layer_2/attention/encdec_output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_2/intermediate/dense/kernel',\n",
       " 'bert/decoder/layer_2/intermediate/dense/bias',\n",
       " 'bert/decoder/layer_2/output/dense/kernel',\n",
       " 'bert/decoder/layer_2/output/dense/bias',\n",
       " 'bert/decoder/layer_2/output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_3/attention/self/query/kernel',\n",
       " 'bert/decoder/layer_3/attention/self/query/bias',\n",
       " 'bert/decoder/layer_3/attention/self/key/kernel',\n",
       " 'bert/decoder/layer_3/attention/self/key/bias',\n",
       " 'bert/decoder/layer_3/attention/self/value/kernel',\n",
       " 'bert/decoder/layer_3/attention/self/value/bias',\n",
       " 'bert/while/decoder/layer_3/attention/self/Softmax',\n",
       " 'bert/decoder/layer_3/attention/output/dense/kernel',\n",
       " 'bert/decoder/layer_3/attention/output/dense/bias',\n",
       " 'bert/decoder/layer_3/attention/output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_3/attention/encdec/query/kernel',\n",
       " 'bert/decoder/layer_3/attention/encdec/query/bias',\n",
       " 'bert/decoder/layer_3/attention/encdec/key/kernel',\n",
       " 'bert/decoder/layer_3/attention/encdec/key/bias',\n",
       " 'bert/decoder/layer_3/attention/encdec/value/kernel',\n",
       " 'bert/decoder/layer_3/attention/encdec/value/bias',\n",
       " 'bert/decoder/layer_3/attention/encdec_output/dense/kernel',\n",
       " 'bert/decoder/layer_3/attention/encdec_output/dense/bias',\n",
       " 'bert/decoder/layer_3/attention/encdec_output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_3/intermediate/dense/kernel',\n",
       " 'bert/decoder/layer_3/intermediate/dense/bias',\n",
       " 'bert/decoder/layer_3/output/dense/kernel',\n",
       " 'bert/decoder/layer_3/output/dense/bias',\n",
       " 'bert/decoder/layer_3/output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_4/attention/self/query/kernel',\n",
       " 'bert/decoder/layer_4/attention/self/query/bias',\n",
       " 'bert/decoder/layer_4/attention/self/key/kernel',\n",
       " 'bert/decoder/layer_4/attention/self/key/bias',\n",
       " 'bert/decoder/layer_4/attention/self/value/kernel',\n",
       " 'bert/decoder/layer_4/attention/self/value/bias',\n",
       " 'bert/while/decoder/layer_4/attention/self/Softmax',\n",
       " 'bert/decoder/layer_4/attention/output/dense/kernel',\n",
       " 'bert/decoder/layer_4/attention/output/dense/bias',\n",
       " 'bert/decoder/layer_4/attention/output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_4/attention/encdec/query/kernel',\n",
       " 'bert/decoder/layer_4/attention/encdec/query/bias',\n",
       " 'bert/decoder/layer_4/attention/encdec/key/kernel',\n",
       " 'bert/decoder/layer_4/attention/encdec/key/bias',\n",
       " 'bert/decoder/layer_4/attention/encdec/value/kernel',\n",
       " 'bert/decoder/layer_4/attention/encdec/value/bias',\n",
       " 'bert/decoder/layer_4/attention/encdec_output/dense/kernel',\n",
       " 'bert/decoder/layer_4/attention/encdec_output/dense/bias',\n",
       " 'bert/decoder/layer_4/attention/encdec_output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_4/intermediate/dense/kernel',\n",
       " 'bert/decoder/layer_4/intermediate/dense/bias',\n",
       " 'bert/decoder/layer_4/output/dense/kernel',\n",
       " 'bert/decoder/layer_4/output/dense/bias',\n",
       " 'bert/decoder/layer_4/output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_5/attention/self/query/kernel',\n",
       " 'bert/decoder/layer_5/attention/self/query/bias',\n",
       " 'bert/decoder/layer_5/attention/self/key/kernel',\n",
       " 'bert/decoder/layer_5/attention/self/key/bias',\n",
       " 'bert/decoder/layer_5/attention/self/value/kernel',\n",
       " 'bert/decoder/layer_5/attention/self/value/bias',\n",
       " 'bert/while/decoder/layer_5/attention/self/Softmax',\n",
       " 'bert/decoder/layer_5/attention/output/dense/kernel',\n",
       " 'bert/decoder/layer_5/attention/output/dense/bias',\n",
       " 'bert/decoder/layer_5/attention/output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_5/attention/encdec/query/kernel',\n",
       " 'bert/decoder/layer_5/attention/encdec/query/bias',\n",
       " 'bert/decoder/layer_5/attention/encdec/key/kernel',\n",
       " 'bert/decoder/layer_5/attention/encdec/key/bias',\n",
       " 'bert/decoder/layer_5/attention/encdec/value/kernel',\n",
       " 'bert/decoder/layer_5/attention/encdec/value/bias',\n",
       " 'bert/decoder/layer_5/attention/encdec_output/dense/kernel',\n",
       " 'bert/decoder/layer_5/attention/encdec_output/dense/bias',\n",
       " 'bert/decoder/layer_5/attention/encdec_output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_5/intermediate/dense/kernel',\n",
       " 'bert/decoder/layer_5/intermediate/dense/bias',\n",
       " 'bert/decoder/layer_5/output/dense/kernel',\n",
       " 'bert/decoder/layer_5/output/dense/bias',\n",
       " 'bert/decoder/layer_5/output/LayerNorm/gamma',\n",
       " 'bert/while/decoder/layer_0/attention/self/Softmax_1',\n",
       " 'bert/while/decoder/layer_1/attention/self/Softmax_1',\n",
       " 'bert/while/decoder/layer_2/attention/self/Softmax_1',\n",
       " 'bert/while/decoder/layer_3/attention/self/Softmax_1',\n",
       " 'bert/while/decoder/layer_4/attention/self/Softmax_1',\n",
       " 'bert/while/decoder/layer_5/attention/self/Softmax_1',\n",
       " 'bert/logits',\n",
       " 'logits']"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or 'logits' in n.name\n",
    "        or 'alphas' in n.name\n",
    "        or 'self/Softmax' in n.name)\n",
    "        and 'adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'global_step' not in n.name\n",
    "        and 'gradients' not in n.name\n",
    "    ]\n",
    ")\n",
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from output/model.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from output/model.ckpt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-29-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-29-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Froze 258 variables.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Froze 258 variables.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Converted 258 variables to const ops.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Converted 258 variables to const ops.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13154 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('output', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.tools.graph_transforms import TransformGraph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "transforms = ['add_default_attributes',\n",
    "             'remove_nodes(op=Identity, op=CheckNumerics, op=Dropout)',\n",
    "             'fold_batch_norms',\n",
    "             'fold_old_batch_norms',\n",
    "             'quantize_weights(fallback_min=-10, fallback_max=10)',\n",
    "             'strip_unused_nodes',\n",
    "             'sort_by_execution_order']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-33-ecde0d9c84a9>:4: FastGFile.__init__ (from tensorflow.python.platform.gfile) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.gfile.GFile.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-33-ecde0d9c84a9>:4: FastGFile.__init__ (from tensorflow.python.platform.gfile) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.gfile.GFile.\n"
     ]
    }
   ],
   "source": [
    "pb = 'output/frozen_model.pb'\n",
    "\n",
    "input_graph_def = tf.GraphDef()\n",
    "with tf.gfile.FastGFile(pb, 'rb') as f:\n",
    "    input_graph_def.ParseFromString(f.read())\n",
    "        \n",
    "inputs = ['Placeholder']\n",
    "transformed_graph_def = TransformGraph(input_graph_def, \n",
    "                                       inputs,\n",
    "                                       ['logits'], transforms)\n",
    "\n",
    "with tf.gfile.GFile(f'{pb}.quantized', 'wb') as f:\n",
    "    f.write(transformed_graph_def.SerializeToString())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_graph(frozen_graph_filename, **kwargs):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, 'rb') as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "\n",
    "    # https://github.com/onnx/tensorflow-onnx/issues/77#issuecomment-445066091\n",
    "    # to fix import T5\n",
    "    for node in graph_def.node:\n",
    "        if node.op == 'RefSwitch':\n",
    "            node.op = 'Switch'\n",
    "            for index in xrange(len(node.input)):\n",
    "                if 'moving_' in node.input[index]:\n",
    "                    node.input[index] = node.input[index] + '/read'\n",
    "        elif node.op == 'AssignSub':\n",
    "            node.op = 'Sub'\n",
    "            if 'use_locking' in node.attr:\n",
    "                del node.attr['use_locking']\n",
    "        elif node.op == 'AssignAdd':\n",
    "            node.op = 'Add'\n",
    "            if 'use_locking' in node.attr:\n",
    "                del node.attr['use_locking']\n",
    "        elif node.op == 'Assign':\n",
    "            node.op = 'Identity'\n",
    "            if 'use_locking' in node.attr:\n",
    "                del node.attr['use_locking']\n",
    "            if 'validate_shape' in node.attr:\n",
    "                del node.attr['validate_shape']\n",
    "            if len(node.input) == 2:\n",
    "                node.input[0] = node.input[1]\n",
    "                del node.input[1]\n",
    "\n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = load_graph('output/frozen_model.pb')\n",
    "x = g.get_tensor_by_name('import/Placeholder:0')\n",
    "logits = g.get_tensor_by_name('import/logits:0')\n",
    "test_sess = tf.InteractiveSession(graph = g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.55 s, sys: 235 ms, total: 2.78 s\n",
      "Wall time: 2.66 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Maisoncelles-la-Jourdan Maisoncelles-la-Jourdan is a commune in the Calvados department of the Basse-Normandie region of northwestern France.'"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "l = test_sess.run(logits, feed_dict = {x: s})\n",
    "encoder.decode([i for i in l[0].tolist() if i > 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = load_graph('output/frozen_model.pb.quantized')\n",
    "x = g.get_tensor_by_name('import/Placeholder:0')\n",
    "logits = g.get_tensor_by_name('import/logits:0')\n",
    "test_sess = tf.InteractiveSession(graph = g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.9 s, sys: 228 ms, total: 6.13 s\n",
      "Wall time: 6.02 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Maisoncelles-la-Jourdan Maisoncelles-la-Jourdan is a commune in the Calvados department of the Basse-Normandie region of northwestern France.'"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "l = test_sess.run(logits, feed_dict = {x: s})\n",
    "encoder.decode([i for i in l[0].tolist() if i > 0])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
